/*
    Copyright (C) 2005 Norbert Klein <norbert@perlprojects.net>

    Copyright (C) 1997-2001  Bradford L. Barrett (brad@mrunix.net)

    Copyright: See COPYING file that comes with this distribution

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version, and provided that the above
    copyright and permission notice is included with all distributed
    copies of this or derived software.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA

    This software uses the gd graphics library, which is copyright by
    Quest Protein Database Center, Cold Spring Harbor Labs.  Please
    see the documentation supplied with the library for additional
    information and license terms, or visit www.boutell.com/gd/ for the
    most recent version of the library and supporting documentation.   
*/



/*
    sitemap.c creates a google sitemap from apache log files. It
    ignores folders and URLs which are listed in the exclude list.
*/

/*********************************************/
/* STANDARD INCLUDES                         */
/*********************************************/

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <string.h>
#include <ctype.h>

#include "output.h"
#include "angolizer.h"                       /* main header */
#include "hashtab.h"
#include "sitemap.h"

#define LINE_LENGTH 2048

char*   getdate_sitemapformat();             /* get current date in sitemap format */
int     getnrlines(FILE*);
int     get_excludelist();
int     get_urllist();
int	sitemapfinal();

char    line[LINE_LENGTH+1];
char**  excludelist;
char**  urllist;
int     diff=0;
time_t  now;                                  /* used by cur_time funct */
char    timestamp[64];
int     first=1;                              /* only run once */
int     nr_excludelines=0;
int     nr_urllist=0;
char    sign='+';


/*************************************************************/
/* CREATE_SITEMAP- create a google sitemap from the log file */
/*************************************************************/

int create_sitemap(u_long u_reg,UNODEPTR *u_array){

   UNODEPTR uptr, *pointer;
   int 	    lenurl=0, exclude=0, inlist=0, i=0;
   FILE     *urllist_fp, *test;
   char     *ln=NULL;
   //run only once
   first=0;

   pointer=u_array;

   //if urllist file and excludelist file dont't exist, create them
   if((test = fopen( sitemapurllist, "r" ))==NULL)
      if((test = fopen( sitemapurllist, "w" ))==NULL) return 1;
   fclose(test);

   if((test = fopen( sitemapexcludefile, "r" ))==NULL)
      if((test = fopen( sitemapexcludefile, "w" ))==NULL) return 1;
   fclose(test);

   //get data from exclude file
   if(!get_excludelist()) return 0;

   //get data from sitemapurllist
   if(!get_urllist()) return 0;
 
   /* open file */
   if ((urllist_fp=(FILE*)fopen(sitemapurllist,"a+"))==NULL) return 1; 

   while (u_reg)
   { 
      uptr=*pointer++;

      if (uptr->flag == OBJ_REG)
      { 
        u_reg--;  
        //skip folders
        lenurl=strlen(uptr->string);
        if(uptr->string[lenurl-1]=='/') continue;

        //skip urls which are part of the exclude list
	exclude=0;
	i=0;

        while(i < nr_excludelines)
        {
            //if single file check for exact match
            if(excludelist[i][strlen(excludelist[i])-1]!='/')
            {
               if(strcmp(uptr->string,excludelist[i])==0)
               {
                  exclude=1; break;
               }
	    }
            else
            {
               if(strstr(uptr->string,excludelist[i])!=NULL)
	       { 
                  exclude=1; break;
               }
            }
	    i++;
	}
        if(exclude==1) continue;

        //check if url is already in urllist file
        inlist=0;
        i=0;
        while(i < nr_urllist)
	{
           if ((ln = (char*)malloc((strlen(uptr->string)+2)*sizeof(char)))==NULL) return 1;
           strcpy(ln,uptr->string);
           strcat(ln,"\n");
	   if(strcmp(ln,urllist[i])==0)
	   {
              inlist=1; break;
           }
	   i++;
           free(ln);
	}
        if(inlist==1) continue;

        //append to urllist file
        fprintf(urllist_fp,"%s\n",uptr->string);
      }
   }
   fclose(urllist_fp);
   //free allocated memory
   while(nr_excludelines--){
      free(excludelist[nr_excludelines]);
   }
   while(nr_urllist--){
      free(urllist[nr_urllist]);
   }  
   free(excludelist);
   free(urllist);   
   
   //sort urllist
   char* strsort;
   if((strsort =(char*) malloc((13+2*strlen(sitemapurllist))*sizeof(char)))==NULL) return 0;
   sprintf(strsort,"sort %s -f -o %s",sitemapurllist,sitemapurllist);
   system(strsort);
   free(strsort);

   //entity escaping
   char* strescape;
   if((strescape =(char*) malloc((35+strlen(sitemapurllist))*sizeof(char)))==NULL) return 0;
   //  \'
   //  shell: sed -i 's/\\\'/\\&apos;/g' filename
   sprintf(strescape,"sed -i \"s/\\\\\\'/\\\\&apos;/g\" %s\n",sitemapurllist);
   system(strescape); 
   // '
   // shell: sed -i 's/\'/\\&apos;/g' filename
   sprintf(strescape,"sed -i \"s/\'/\\\\&apos;/g\" %s\n",sitemapurllist);
   system(strescape); 
   
   free(strescape);

   if(sitemapfinal()==0) return 0;
   return 1;
}



/**************************************************************************************/
/* GETDATE_SITEMAPFORMAT- create the date/time string according to the sitemap format */
/**************************************************************************************/

char* getdate_sitemapformat(){

   time_t now = time(NULL);
   struct tm* ptrgmt;
   struct tm* ptrloc;
   struct tm* tgmt;
   struct tm* tloc;

   tgmt=(struct tm*)gmtime(&now);

   ptrgmt = (struct tm*) malloc(sizeof(struct tm));

   if(ptrgmt==NULL)
   {
      return 0;
   }
   ptrgmt->tm_sec=tgmt->tm_sec;
   ptrgmt->tm_min=tgmt->tm_min;
   ptrgmt->tm_hour=tgmt->tm_hour;
   ptrgmt->tm_mday=tgmt->tm_mday;
   ptrgmt->tm_mon=tgmt->tm_mon;
   ptrgmt->tm_year=tgmt->tm_year;

   tloc=(struct tm*)localtime(&now);
   ptrloc = (struct tm*) malloc(sizeof(struct tm));
   if(ptrloc==NULL)
   {
      return 0;
   }
   ptrloc->tm_sec=tloc->tm_sec;
   ptrloc->tm_min=tloc->tm_min;
   ptrloc->tm_hour=tloc->tm_hour;
   ptrloc->tm_mday=tloc->tm_mday;
   ptrloc->tm_mon=tloc->tm_mon;
   ptrloc->tm_year=tloc->tm_year;

   // calculate the difference to gmt in hours
   // google did not accept the localtime - difference to gmt, so
   // we use gmt+0000 for the time being
   /*
   if (local_time){
      strftime(timestamp,sizeof(timestamp),"%Y-%m-%dT%H:%M:%S",tloc);
 
      //epoch=jdate(1,1,1970);  
      u_long gmt_s=0.0;
      u_long loc_s=0.0;
      long diff_s=0.0;
      gmt_s=((jdate(ptrgmt->tm_mday,ptrgmt->tm_mon,ptrgmt->tm_year)-epoch)*86400)+
                      (ptrgmt->tm_hour*3600)+(ptrgmt->tm_min*60)+ptrgmt->tm_sec;
      loc_s=((jdate(ptrloc->tm_mday,ptrloc->tm_mon,ptrloc->tm_year)-epoch)*86400)+
                      (ptrloc->tm_hour*3600)+(ptrloc->tm_min*60)+ptrloc->tm_sec;
      diff_s=loc_s-gmt_s;
      diff=diff_s/3600;
      if (diff>0) sign='-';
      else {diff=-diff; sign='+';}

   }
   else
   {
   */
      strftime(timestamp,sizeof(timestamp),"%Y-%m-%dT%H:%M:%S",ptrgmt);   
      diff=0;
   //}


   free(ptrgmt);
   free(ptrloc);
   return timestamp;
}


/****************************************************************/
/* GET_EXCLUDELIST- load all URLs from exclude file into memory */
/****************************************************************/

int get_excludelist(){

   int nr=0,i=0;
   FILE *exclude_fp;

   // open exclude file
   if ((exclude_fp=(FILE*)fopen(sitemapexcludefile,"r"))==NULL) return 0; 

   //count number of URL lines
   nr=getnrlines(exclude_fp);

   if ((excludelist = (char**)malloc(nr*sizeof(char*)))==NULL) return 0;
   i=0;
   int len=0;
   while (nr--)
   {
      fgets(line, LINE_LENGTH, exclude_fp);
      len=strlen(line);

      /* skip comments and blank lines */
      if ((line[0]=='#') || isspace((int)line[0])) continue;
    
      if ((excludelist[i] = (char *)malloc((len)*sizeof(char))) == NULL) return 0;
      memcpy(excludelist[i],line,len);
      excludelist[i][len-1]='\0';
      i++;
   }
   nr_excludelines=i;
   fclose(exclude_fp);

   return 1; 
}



/****************************************************************/
/* GET_URLLIST- load all URLs from old urllist file into memory */
/****************************************************************/

int get_urllist(){

   int nr=0,i=0;
   FILE* urllist_fp;

   // open exclude file
   if ((urllist_fp=(FILE*)fopen(sitemapurllist,"r"))==NULL) return 0; 

   //count number of URL lines
   nr=getnrlines(urllist_fp);

   if ((urllist = (char**)malloc(nr*sizeof(char*)))==NULL) return 0;
   i=0;
   int len=0;
   while (nr--)
   {
      fgets(line, LINE_LENGTH, urllist_fp);
      len=strlen(line);

      /* skip comments and blank lines */
      if ((line[0]=='#') || isspace((int)line[0])) continue;

      if ((urllist[i] = (char *)malloc((len+1)*sizeof(char))) == NULL) return 0;
      strcpy(urllist[i],line);
      i++;
   }
   nr_urllist=i;
   fclose(urllist_fp);
   return 1; 
}



/**************************************************************************************/
/* GETNRLINES- get the number of lines in a file                                      */
/**************************************************************************************/

int getnrlines(FILE* fname){
   int c, nc = 0, nl = 0;
   while ((c = fgetc(fname)) != EOF )
   {
      nc++;
      if (c == '\n') nl++;
   }
   rewind(fname);
   return nl;
}



/************************************************************************************/
/* SITEMAPFINAL- create the sitemap by using all URLs from the sitemapurllist-file  */
/************************************************************************************/

int sitemapfinal(){

   int nr=0, len=0;
   FILE *out_fp,*allurls_fp;
   char *protocol,*linex;
   
   //create empty file or erase content
   if ((out_fp=(FILE*)fopen(sitemappathfname,"w"))==NULL) return 0; 

   //read the whole new list of urls from urllist file
   if ((allurls_fp=(FILE*)fopen(sitemapurllist,"r"))==NULL) return 0; 

   //count number of URL lines
   nr=getnrlines(allurls_fp);

   char *date_sitemapformat=getdate_sitemapformat();

   if (use_https)
   {
      protocol="https://";
   }
   else
   {
      protocol="http://";
   }

  /* write header to sitemap */
   fprintf(out_fp,"<?xml version=\"1.0\" encoding=\"%s\"?>\n",sitemapencoding);
   fprintf(out_fp,"<urlset xmlns=\"http://www.google.com/schemas/sitemap/0.84\">\n");

   while (nr--)
   {
      fgets(line, LINE_LENGTH, allurls_fp);
      /* skip comments and blank lines */
      if ((line[0]=='#') || isspace((int)line[0])) continue;

      len=strlen(line);
      if ((linex = (char *)malloc((len)*sizeof(char))) == NULL) return 0;
      memcpy(linex,line,len);
      linex[len-1]='\0';

      fprintf(out_fp,"<url>\n");
      fprintf(out_fp,"<loc>%s%s%s</loc>\n", protocol,hname,linex);
      fprintf(out_fp,"<lastmod>%s%c%02d:00</lastmod>\n",date_sitemapformat,sign,diff);
      fprintf(out_fp,"<changefreq>%s</changefreq>\n",sitemapchangefreq);
      fprintf(out_fp,"<priority>%s</priority>\n",sitemappriority);
      fprintf(out_fp,"</url>\n");
      free(linex);
   }
   fprintf(out_fp,"</urlset>\n");

   fclose(allurls_fp);
   fclose(out_fp);
   
   //gzip sitmap
   char* strgzip;
   if((strgzip =(char*) malloc((9+strlen(sitemappathfname))*sizeof(char)))==NULL) return 0;
   sprintf(strgzip,"gzip -f %s",sitemappathfname);
   system(strgzip);
   free(strgzip);
   
   return 1;
}
